package code.reptile.main;

import com.sun.media.sound.InvalidFormatException;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import code.reptile.core.LinkQueue;

/**
* <p></p>
*
* @version 1.0
* @author:huangwf
* @date:2017/8/18 11:42
* @history
*/

public class GanJiReptileList {

private final static String TEXTPRTHNAME = "ReptilExcel/out/excel/city.xls";
private final static String TEXTPRTHNAME_ONE = "ReptilExcel/out/excel/area.xls";
private final static String FILEPATH = "ReptilExcel\\out\\html\\ganjilist\\";
public int AllCount = 0;

public static void main(String[] args) {
String url = "http://www.ganji.com/index.htm";

String path = null;
//                path = DownLoadFile.downloadFile(url,filePath);
path = "ReptilExcel/out/html/ganji/www.ganji.com_index.htm.html";
analzeLocalHtml(path);
}

//解析本地的html
private static void analzeLocalHtml(String path) {

//读取本地html的路径
File file = new File(path);
//生成一个数组用来存储这些路径下的文件名
try {
if (file.isFile()) {
System.out.println("正在解析本地：" + file.getName());

//下面开始解析本地的html
Document doc = Jsoup.parse(file, "UTF-8");
//得到html的所有东西
Elements links = doc.getElementsByClass("all-city");
//分离出html下<td>...</td>之间的所有东西
String href = null;
String text = null;
Element link = links.get(1);
Elements citys = link.getElementsByTag("a");
forRead(citys, TEXTPRTHNAME, null, 0);
while (!LinkQueue.unVisitedUrlsEmpty() && LinkQueue.getVisitedUrlNum() <= 1000) {
String visitUrl = (String) LinkQueue.unVisitedUrlDeQueue();
if (visitUrl == null)
continue;
//                    System.out.println("访问：" + visitUrl);
LinkQueue.addVisitedUrl(visitUrl);
//                    System.out.println(visitUrl.substring(0,visitUrl.length()-1));
//                                        DownLoadFile.downloadFile(visitUrl, FILEPATH);
Get_Url(visitUrl);
}
}
} catch (Exception e) {
System.out.println("网址：" + file.getName() + "解析出错");
e.printStackTrace();
}
}

private static void analzeStringHtml(String html, String url) throws IOException, InvalidFormatException {
System.out.println("正在解析网址：....." + url);
System.out.println(html);
//下面开始解析html
Document doc = Jsoup.parse(html);
//得到html的所需要的东西
Elements links = doc.getElementsByClass("f-recruit-area");

String href = null;
String text = null;
for (Element link : links) {
//分离出html下<a>...<a>之间的所有东西
    Elements areas = link.getElementsByTag("a");
    forRead(areas, TEXTPRTHNAME_ONE, url, 1);
    }
    }

    private static void forRead(Elements elements, String pathName, String url, int tag) throws IOException, InvalidFormatException {

    List<Map<String, String>> list = new ArrayList<Map<String, String>>();
    for (Element area : elements) {
    Map<String, String> areMap = new HashMap<String, String>();
    String uri = area.attr("href");
    areMap.put("cityName", area.text());
    areMap.put("cityHref", uri);
    LinkQueue.addUnvisitedUrl(uri + "zhaopin/");
    list.add(areMap);
    if (list.size() > 499) {
    for (Map item : list) {
    item.clear();
    }
    list.clear();
    }
    }

    }

    private static int Get_Url(String url) {
    System.out.println("访问网址：" + url);
    int isSuccess = 0;
    HttpClient httpClient = new HttpClient();
    // 设置 Http 连接超时 5s
    httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(
    5000);
    /* 2.生成 GetMethod 对象并设置参数 */
    GetMethod getMethod = new GetMethod(url);
    // 设置 get 请求超时 5s
    getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
    // 设置请求重试处理
    getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
    new DefaultHttpMethodRetryHandler());

    /* 3.执行 HTTP GET 请求 */
    try {
    int statusCode = httpClient.executeMethod(getMethod);
    // 判断访问的状态码
    if (statusCode != HttpStatus.SC_OK) {
    System.err.println("Method failed: "
    + getMethod.getStatusLine());
    } else {
    isSuccess = 1;
    }
    /* 4.处理 HTTP 响应内容 */
    String responseBody = getMethod.getResponseBodyAsString();// 读取为字节数组
    analzeStringHtml(responseBody, url);
    return isSuccess;

    } catch (HttpException e) {
    // 发生致命的异常，可能是协议不对或者返回的内容有问题
    System.out.println("Please check your provided http address!");
    e.printStackTrace();
    } catch (IOException e) {
    // 发生网络异常
    e.printStackTrace();
    } finally {
    // 释放连接
    getMethod.releaseConnection();
    }
    return isSuccess;
    }
    }
